# coding=utf-8
# 爬取网页图片
import re
import requests
import bs4
import os
import save_html

url = 'https://www.58pic.com/?tid=1219918&utm_source=baidu2&utm_medium=cpc&utm_campaign=%C6%B7%C5%C6&utm_content=%C7%A7%CD%BC&utm_term=%C7%A7%CD%BC%CD%F8+&sdclkid=A5f615ei152sbrDRxg'
HEADER = {  # 下载图片时不加这个报403
    "USER-AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"
}
# save_html.save_html(url, 'qiantu.html')
try:
    htmlfile = requests.get(url)
except Exception as err:
    print("网页读取失败 %s" % err)

objSoup = bs4.BeautifulSoup(htmlfile.text, 'lxml')
img_list = list(set(objSoup.select('img')))  # 去重
# 保存下载失败的图片
err_list = []
# 保存下载成功的图片
success_list = []
img_destdir = 'imglists2'
if os.path.exists(img_destdir) == False:
    os.mkdir(img_destdir)
print(img_list, '\n', len(img_list))
if len(img_list) > 0:
    for i in img_list:
        imgUrl = i.get('data-original') or i.get('src') or i.get('data-src')
        if imgUrl == None:
            print('none:', i)
            err_list.append(i)
            continue
        print('%s 图片下载中 ...' % imgUrl)
        finUrl = 'https:' + imgUrl
        print(finUrl, imgUrl, i)
        try:
            picture = requests.get(finUrl, headers=HEADER)  # 下载图片
        except Exception as err:
            print("图片下载失败；", err)
            err_list.append(i)
            continue
        # picture.raise_for_status()                          # 验证图片是否下载成功
        print('%s 图片下载成功' % finUrl)
        pattern = '.*(\.png|\.jpg)$'
        saveUrl = re.search(pattern, finUrl)  # 判断url是否以图片类型的后缀结尾，如果不是，拼接上，用于产生正确的图片格式
        if saveUrl == None:
            saveUrl = finUrl + '.png'
        else:
            saveUrl = saveUrl.group()
        print('saveurl：', saveUrl)
        # 先开启文件，再存储图片
        pictFile = open(os.path.join(img_destdir, os.path.basename(saveUrl)), 'wb')
        success_list.append(i)
        for diskStorage in picture.iter_content(10240):
            pictFile.write(diskStorage)
        pictFile.close()  # 关闭文件

print('error_list:', err_list, '\n', len(err_list))
print('success_list:', success_list, '\n', len(success_list))
print(type(objSoup), objSoup.title, img_list)
